In [29]:
from sklearn.datasets import load_iris
from scipy.stats import spearmanr
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from statsmodels.stats.multitest import multipletests
iris_df = data_ok[['poptot', 'gdpppp', 'moy_gini', 'incomeLevel']]
def get_correlations(df):
df = df.dropna()._get_numeric_data()
dfcols = pd.DataFrame(columns=df.columns)
pvalues = dfcols.transpose().join(dfcols, how="outer")
correlations = dfcols.transpose().join(dfcols, how="outer")
for ix, r in enumerate(df.columns):
for jx, c in enumerate(df.columns):
sp = spearmanr(df[r], df[c])
correlations[c][r] = sp[0]
pvalues[c][r] = sp[1] if ix > jx else np.nan # Only store values below the diagonal
return correlations.astype("float"), pvalues.astype("float")
correlations, uncorrected_p_values = get_correlations(iris_df)
# Correct p-values for multiple testing and check significance (True if the corrected p-value < 0.05)
shape = uncorrected_p_values.values.shape
significant_matrix = multipletests(uncorrected_p_values.values.flatten())[0].reshape(
shape
)
# Here we start plotting
g = sns.clustermap(correlations, cmap="vlag", vmin=-1, vmax=1)
# Here labels on the y-axis are rotated
for tick in g.ax_heatmap.get_yticklabels():
tick.set_rotation(0)
# Here we add asterisks onto cells with signficant correlations
for i, ix in enumerate(g.dendrogram_row.reordered_ind):
for j, jx in enumerate(g.dendrogram_row.reordered_ind):
if i != j:
text = g.ax_heatmap.text(
j + 0.5,
i + 0.5,
"*" if significant_matrix[ix, jx] or significant_matrix[jx, ix] else "",
ha="center",
va="center",
color="black",
)
text.set_fontsize(20)
